type token =
   | Unknown_symbol of string
   | Unexpected_char of char
   | Unexpected_eof

   | Nil
   | True
   | False

   | Eq
   | Ne
   | Lt
   | Gt
   | Le
   | Ge

   | Not
   | And
   | Or
   | Xor

   | Lsh
   | Rsh

   | If
   | Match
   | Switch
   | Case
   | When
   | Else

   | Break
   | Continue
   | Return

   | Lbrace
   | Rbrace
   | Lparen
   | Rparen
   | Lbracket
   | Rbracket
   | Langle
   | Rangle
   | Comma
   | Dot
   | Colon
   | Terminator

   | Assign
   | Plus
   | Minus
   | Star
   | Divide
   | Mod
   | Power

   | Integer of int
   | Real of float
   | Imaginary of float
   | String of string

   | Identifier of string
   | Member of string
   | Type of string

   | This
   | Super

   | Import
   | Enum
   | Class
   | Interface
   | Includes
   | Extends
   | Implements
   | Where

   | Literal
   | Protected
   | Package
   | Public
   | New
   | Results
   | Throws

   | Var

let keywords:(string, token)Hashtbl.t = Hashtbl.create 30
let symbols:(string, token)Hashtbl.t = Hashtbl.create 30
let symbol_fragments:(string, bool)Hashtbl.t = Hashtbl.create 100

let get_identifier id =
   let c = String.get id 0 in
      if (c <> '_') & (c = Char.uppercase c) then
         (* a type can't contain an underscore *)
         if String.contains id '_' then
            Unknown_symbol id
         else
            Type id
      else
         (* an identifier can't contain an uppercase letter *)
         if id <> String.lowercase id then
            Unknown_symbol id
         else
            if c = '_' then
               (* a member starts with an underscore *)
               Member id
            else
               (* could be a keyword or an identifier *)
               try Hashtbl.find keywords id with Not_found -> Identifier id

let get_symbol sym =
   try Hashtbl.find symbols sym with Not_found -> Unknown_symbol sym

let is_symbol_fragment sym =
   try Hashtbl.find symbol_fragments sym with Not_found -> false

let init =
   Hashtbl.add keywords "nil" Nil;
   Hashtbl.add keywords "true" True;
   Hashtbl.add keywords "false" False;

   Hashtbl.add keywords "eq" Eq;
   Hashtbl.add keywords "ne" Ne;
   Hashtbl.add keywords "lt" Lt;
   Hashtbl.add keywords "gt" Gt;
   Hashtbl.add keywords "le" Le;
   Hashtbl.add keywords "ge" Ge;

   Hashtbl.add keywords "import" Import;

   let rec add_sym sym tok =
      Hashtbl.add symbols sym tok;
      add_sym_frag sym
   and add_sym_frag sym =
      let len = String.length sym in
         for i = 1 to len do
            let frag = String.sub sym 0 i in
               Hashtbl.replace symbol_fragments frag true
         done
      in
      add_sym "=" Assign;
      add_sym "+" Plus;
      add_sym "-" Minus;
      add_sym "*" Star;
      add_sym "/" Divide;
      add_sym ">" Rangle;
      add_sym "->" Results;
   ;;

let print tok =
   begin
      try match tok with
         | Unknown_symbol s ->
            print_string "unknown symbol ";
            print_string s;
         | Unexpected_char c ->
            print_string "unexpected char ";
            print_char c;
         | Unexpected_eof ->
            print_string "unexpected eof";
         | Nil -> print_string "nil";
         | True -> print_string "true";
         | False -> print_string "false";
         | Eq -> print_string "eq";
         | Ne -> print_string "ne";
         | Integer i ->
            print_string "integer ";
            print_string (string_of_int i);
         | Real i ->
            print_string "real ";
            print_string (string_of_float i);
         | Imaginary i ->
            print_string "imaginary ";
            print_string (string_of_float i);
         | String s ->
            print_string "string ";
            print_string s;
         | Identifier s ->
            print_string "identifier ";
            print_string s;
         | Member s ->
            print_string "member ";
            print_string s;
         | Type s ->
            print_string "type ";
            print_string s;
         | Rangle -> print_string ">";
         | Results -> print_string "->";
         | _ ->
            print_string "some token";
      with Stream.Error s ->
         print_string "error: ";
         print_string s;
   end;
   print_newline ()
